
#include "site.h"
#include "main.h"
#include "database.h"
#include "site.h"
#include "wcurl.h"
#include <openssl/md5.h>
#include <pcrecpp.h>
#include "glb.h"

//-----------------------------------------------------------------------------
void replace_all(string &content, const string &from, const string &to)
{
    size_t i = 0;
    size_t j;
    while((j = content.find(from, i)) != string::npos)
    {
        content.replace(j, from.size(), to);
        i = j + to.size();
    }
}


void __normalize(string& sname)
{
    if(sname.find("http://")!=0)
    {
        GLOGW("Site '"<<sname<<"' is missing protocol: http:// assumed!")
        sname=string("http://") + sname; // perhaps
    }
    replace_all(sname,"www.","");
}

//-----------------------------------------------------------------------------
string &strip_tags(string &html)
{
    bool inflag = false;
    bool done = false;
    size_t i, j;
    while (!done)
    {
        if (inflag)
        {
            i = html.find('>');
            if(i==(size_t)-1)
            {
                if (i != string::npos)
                {
                    inflag = false;
                    html.erase(0, i+1);
                }
                else
                {
                    done = true;
                    html.erase();
                }
            }
        }
        else
        {
            i = html.find('<');
            if (i != string::npos)
            {
                j = html.find('>');
                if (j == string::npos)
                {
                    inflag = true;
                    done = true;
                    html.erase(i);
                }
                else
                {
                    html.erase(i, j-i+1);
                }
            }
            else
            {
                done = true;
            }
        }
    }
    return html;
}

//-----------------------------------------------------------------------------
uripage::uripage(ref_count<wsite> site, string& uri, const uripage* root):
    _web_site(site),
    _link_id(0),
    _parent_id(0),
    _bkls(0),
    _deleted(false),
    _links_count(0),
    _updated(false),
    _accum(0),
    _p_dbase(0),
    _link_name("index"),
    _level_down(0)
{
    if(root)
    {
        size_t  r=uri.find(root->_proto);
        if(r==string::npos)
        {
            _uri=root->_proto;

            if(!root->_port.empty())
            {
                _uri+=":"+root->_port;
            }
            if(uri.find(root->_host)==string::npos)
            {
                if(!uri.empty())
                    _uri+=root->_host+"/";
                else
                    _uri+=root->_host;
            }
        }
        replace_all(uri,"www.","");
        _uri=uri;
    }
    else //provded  by interface
    {
        __normalize(uri);
        _uri = uri;
    }

    string::iterator b = _uri.begin();
    string* pwhat = &_proto;
    for(; b != _uri.end(); ++b)
    {
        switch(*b)
        {
        case ':':
            if(_host.empty())
            {
                assert(!_proto.empty());
                _proto+="://";
                b+=2;
                pwhat=&_host;
            }
            else
            {
                assert(!_proto.empty());
                assert(!_host.empty());
                pwhat=&_port;
                _port+=":";//put in the port so we dont test it
            }
            break;
        case '/':
            if(_path.empty())
                pwhat=&_path;
            else
                pwhat->append(1,*b);
            break;
        case '?':
            pwhat=&_qry;
            pwhat->append("?"); //
            break;
        default:
            pwhat->append(1,*b);
            break;
        }
    }
    if(_uri[_uri.length() - 1 ]=='/' )
    {
        _uri = _uri.substr(0,_uri.length() - 1);
    }
    _hostname = _proto + _host + _port;
}   // Pars


//-----------------------------------------------------------------------------
void uripage::create( mysqlcls* db)
{
    //wsite* ps = _web_site.ptr();

    assert(0==_p_dbase);
    _p_dbase = db;

    if(_web_site->_reindex && _parent_id==0)
    {
        db_remove();
    }
    if(!_robots_omit())
    {
        if(_parent_id==0)
        {
            _db_add_site();
        }
        _index(false);
    }
    else
    {
        GLOGW("Robiots reject:" << _uri);
    }
}

//-----------------------------------------------------------------------------
// removes site ids where uri=_uri.
void uripage::db_remove()
{
    stringstream    s;

    s << "SELECT s_id FROM wss WHERE sn='"<<_uri<<"';";
    SR rezs(_p_dbase->qry(s, true));
    if(rezs->rows())
    {//
        GLOGI("deleting " << _uri <<"\n");

        int32_t s_id = _I(rezs->fetch(0));
        s.str("");
        s << "SELECT l_id FROM lnks WHERE s_id="<<s_id;
        SR rezl(_p_dbase->qry(s, true));
        MYSQL_ROW r;
        while((r = rezl->fetch())>0)
        {
            char tbl_name[32];
            int32_t l_id = _I(r[0]);
            for(int32_t tbl = 0; tbl < _TABLES; ++tbl)
            {
                ::sprintf(tbl_name,"kwds%d",tbl);
                s.str("");
                s << "DELETE FROM "<<tbl_name<<" WHERE l_id="<<l_id;
                SR ldel(_p_dbase->qry(s, false));
            }
        }
        rezl.detach();

        s.str("");
        s << "DELETE FROM lnks WHERE s_id="<<s_id;
        _p_dbase->qry(s);

        s.str("");
        s << "DELETE FROM wss WHERE s_id="<<s_id;
        _p_dbase->qry(s);
    }
}

//-----------------------------------------------------------------------------
// add or update a website
bool uripage::_db_add_site()
{
    bool         rv=false;
    stringstream s;
    const htpargs& hs = _web_site->_hargs;

    //wsite* ps = _web_site.ptr();
    string sname=hs["sn"];

    __normalize(sname);
    if(sname.empty())
    {
        return false;
    }
    if(sname[sname.length()-1]=='/')
    {
        sname = sname.substr(0,sname.length() - 1);
    }

    s << "SELECT s_id FROM wss WHERE sn="<<_S(sname);
    SR rezs(_p_dbase->qry(s, true));
    size_t rows = rezs->rows();
    if(rows)
    {
       // assert(rezs->rows() == 1); //uniique page
       if(rows !=1 )
       {
           GLOGE("FATAL");
       }
        _web_site->_site_id = _U(rezs->fetch(0));
        //update
        s.str("");
        s <<"UPDATE wss SET" <<
            " z_id="<<_web_site->_z_id<<
            ", sn="<<_S(sname)<<
            ", tit="<<STR("tit")<<
            ", dsc="<<STR("dsc")<<
            ", idat=NOW()"<<
            ", mxd="<<INT("mxd")<<
            ", rqt="<<STR("rqt")<<
            ", dit="<<STR("dit")<<
            ", offs="<<INT("offs") <<
            ", ck="<<INT("ck")<<
            ", lg="<<INT("lg")<<
            ", pri="<<INT("pri")<<
            " WHERE s_id="<<_web_site->_site_id<<";";
        _p_dbase->qry(s);
    }
    else
    {
        s.str("");

        s << "INSERT INTO wss (z_id,sn,tit,dsc,idat,mxd,rqt,dit,offs,ck,lg,pri) " <<
        " VALUES ("<<_web_site->_z_id<<","<<_S(sname)<<","<<STR("tit")<<","<<STR("desc")<<",NOW(),"<< hs("mxd")<<
        ","<<STR("rqt")<<","<<STR("dit")<<","<<hs("offs")<<","<<hs("ck")<<","<<hs("lg")<<
        ","<<hs("pri") << ");";
        _p_dbase->qry(s);
        _web_site->_site_id = _p_dbase->lastid();
        rv=true;
    }
    return rv;
}

// light parse robots. do it here instead other out there faulty implemetation
// at least i fix it if someting is not right
bool  uripage::_robots_omit()
{
    if(_robots.empty())
    {
        string urldoc = _hostname + "/robots.txt";
        wcurl        request(urldoc);
        string       robots = request.header();

        if(robots.find("404") != string::npos)
        {
            return false; //allowed
        }

        request.reuse();
        _robots = request.content();
    }

    stringstream ss(_robots);
    string s;
    bool   section = false;
    while (getline(ss, s, '\n'))
    {
        bool bua = s.find("User-agent")!=string::npos;
        if(bua)
        {
            section = ((s.find("*")!=string::npos ||s.find(_AGENT)!=string::npos ));
            continue;
        }
        if(section)
        {
            if(_path == "/")
            {
                if (s=="Dissalow: /")
                {
                    return true;
                }
                continue;
            }
            else if(s.find(_path)!=string::npos)
            {
                if(s.find("Dissalow")!=string::npos)
                {
                    return true;
                }
                continue;
            }
        }
    }
    return false;
}

bool uripage::_moved()
{
    stringstream    ss(_header);
    string          s;
    _mime          = "text";
    while (getline(ss, s, '\r'))
    {
        if(s=="\n")
            break;    //header done
        else if(s.find("404")!=string::npos) // no link_page
        {
            ++_bkls;
            _deleted=true;
            return true;
        }
        if(s.find("Location:")!=string::npos ) // redirect
        {
            _location = s.substr(11);
            if(!_location.empty())
            {
                uripage pl(_web_site, _location, 0);

                if(pl._uri != _uri)
                {
                    GLOGT(_uri <<" Moved to " << pl._uri);
                    return true;
                }
            }
            return false;
        }
    }

    return false;
}
/*
        NEW_LINK,
        REFR_LINK,
        CHNG_LINK,
        PG_DONE,
*/
uripage::PAGE_ST uripage::_changed()
{
    stringstream    s;

//    wsite* ps = _web_site.ptr();

    s << "SELECT l_id, md5, z_id, ldat FROM lnks WHERE s_id="<<_web_site->_site_id<<" AND url="<<_S(_uri);
    SR rezs(_p_dbase->qry(s, true));
    if(rezs->rows() == 0)
    {
        return NEW_LINK; // not even in DB
    }

    //_web_site->_total_links=rezs->rows();

    //page date from header
//    int cols = rezs->rows();
    assert(rezs->rows() == 1);
    MYSQL_ROW row = rezs->fetch();

    _link_id = _U(row[0]);
    if(_web_site->_z_id == (uint32_t)_I(row[2]))
    {
        GLOGW(_uri << ": parsed in this sess. SKIPPED");
        return PG_DONE;
    }

    const char* pdate = row[3];
    if(_date == pdate) //date same
    {
        GLOGW(_uri << ": page date did no changed. SKIPPED");
        return DB_LINK;
    }

    // page MD5 from content
    wcurl   cntn(_uri);
    _content = cntn.content();

    GLOGT("FETCING PAGE: *********** ####### " << _uri << " ######  **************");

    unsigned char result[MD5_DIGEST_LENGTH];
    _md5="";
    char out[8];
    MD5((const unsigned char*)_content.c_str() , _content.length(), result);
    for(int k=0; k<MD5_DIGEST_LENGTH; k++)
    {
        sprintf(out,"%02X", (char)result[k]);
        _md5+=out;
    }
//    const char* m = row[1];
    if(_md5 == row[1])
    {
        GLOGW( _uri << ": page md5/crc did no changed. SKIPPED");
        return DB_LINK;
    }
    return NEW_LINK;
}

// page did not chnage so add for subsequest nlks from the
// things we have in the db
void uripage::_add_links_from_dbpage()
{
//    wsite* ps = _web_site.ptr();

    stringstream s;
    s << "SELECT url, name, dpth FROM lnks WHERE s_id="<<_web_site->_site_id<<" AND p_id="<<_link_id;

    SR rezl(_p_dbase->qry(s, true));
    MYSQL_ROW row;
    while((row = rezl->fetch()))
    {
        _enqueue(row[0], row[1], _web_site->_z_id, _I(row[2]), _link_id);
    }
}

void uripage::_index(bool isnewsite)
{
    if(_header.empty())
    {
        wcurl   hdr(_uri);
        _header = hdr.header();

        GLOGD("Header:" << _header);

        _mime   = hdr._mime;
        _date   = hdr._date;
    }
    if(_mime.find("text")==string::npos)
    {
        return; // no text
    }

    if(_moved()) // this parsed the header entities
    {
        db_remove();
        if(_deleted)
            return;
        if(!_location.empty())
        {
            pcrecpp::RE("[\r\n]").GlobalReplace("", &_location);
            if(!_location.empty())
                _enqueue(_location, "", _web_site->_z_id, _level_down, _parent_id);
        }
        return;
    }
    PAGE_ST state = NEW_LINK;

    // if(!isnewsite)
    {
        state = _changed();
    }

    if(state==PG_DONE)
    {
        //parsed here
        return;
    }

    if(state==DB_LINK)
    {
        //page link did not change
        _db_update_link(true);
        _add_links_from_dbpage();
        return;
    }


    if(_content.empty())
    {
        wcurl   cntn(_uri);
        _content = cntn.content();

        unsigned char result[MD5_DIGEST_LENGTH];
        MD5((const unsigned char*)_content.c_str() , _content.length(), result);
        char out[8];
        _md5.clear();
        for(int k=0; k<MD5_DIGEST_LENGTH; k++)
        {
            sprintf(out,"%02X", (char)result[k]);
            _md5+=out;
        }
    }
    if(!_content.empty())
    {
        _parse_page();
    }
}

static bool __bad_link(const string & s)
{
    /*
    if(s.find("(")!=string::npos)    return true;
    if(s.find(".zi")!=string::npos)  return true;
    if(s.find(".ra")!=string::npos)  return true;
    if(s.find(".ta")!=string::npos)  return true;
    if(s.find(".gz")!=string::npos)  return true;
    if(s.find(".wm")!=string::npos)  return true;
    if(s.find(".fl")!=string::npos)  return true;
    if(s.find(".mp")!=string::npos)  return true;
    if(s.find(".vm")!=string::npos)  return true;
    if(s.find(".og")!=string::npos)  return true;
    if(s.find(".pd")!=string::npos)  return true;
    if(s.find(".xm")!=string::npos)  return true;
    if(s.find(".jp")!=string::npos)  return true;
    if(s.find(".pn")!=string::npos)  return true;
    if(s.find(".gi")!=string::npos)  return true;
    if(s.find(".bm")!=string::npos)  return true;
    if(s.find(".ic")!=string::npos)  return true;
*/

    if(s.find(".ht")==string::npos)  return false;
    if(s.find(".as")==string::npos)  return false;
    if(s.find(".ph")==string::npos)  return false;
    if(s.find(".cg")==string::npos)  return false;
    if(s.find(".tx")==string::npos)  return false;
    if(s.find(".")==string::npos)  return false;
    return true;
}

int uripage::_db_update_link(bool update)
{
//    wsite* ps = _web_site.ptr();
    stringstream s;

    if(_link_id==0)
    {
        assert(update==false); //called to update only. cheking flag

        s << "INSERT INTO lnks (s_id, z_id, p_id, url, tit, dsc, name,txt, idat, size, md5,vis,dpth,nlks,blks,ldat) VALUES (" <<
        _web_site->_site_id <<","<<_web_site->_z_id <<","<< _parent_id <<","<< _S(_uri) <<","<< _S(_title) <<","<< _S(_desc) <<","<<
        _S(_link_name) <<",'content'" << ",NOW()," << 0 <<","<<
        _S(_md5) <<","<< 1 <<","<< _level_down <<","<< _links_count << "," << _bkls << ","<< _S(_date) <<");";

        _p_dbase->qry(s);
        _link_id = _p_dbase->lastid();
    }
    else
    {
        s << "UPDATE lnks SET" <<
        " s_id="<<_web_site->_site_id<<
        ", z_id="<<_web_site->_z_id<<
        ", ldat="<<_S(_date)<<
        " WHERE l_id="<<_link_id;
        _p_dbase->qry(s);
    }
    if(_p_dbase->err()==0)
        assert(_link_id>0);
    return     _p_dbase->err();
}

inline int32_t __calc_wkey(const string& word)
{
    int32_t sum = 0;
    for(size_t i=0; i<word.length(); ++i)
    {
        sum += (unsigned char)word[i];
    }
    return (sum % _TABLES);
}


int uripage::_db_finalize_page()
{

    char* from = new char[_compressed.length() * 3 + 3];
    mysql_real_escape_string(&_p_dbase->_mysql, from, _compressed.c_str(), _compressed.length());

    char sdesc[256]=" ";
    char stitle[256]=" ";
    if(!_desc.empty())
        mysql_real_escape_string(&_p_dbase->_mysql, sdesc, _desc.c_str(), min(_desc.length(),size_t(256/3)-1));
    if(!_title.empty())
        mysql_real_escape_string(&_p_dbase->_mysql, stitle, _title.c_str(), min(_title.length(),size_t(256/3)-1));


    stringstream s;
    s << "UPDATE lnks SET" <<
    " nlks="<<_links_count<<
    ", blks="<<_bkls<<
    ", txt="<<_S(from)<<
    ", tit="<< _S(stitle)<<
    ", dsc="<<_S(sdesc)<<
    ", name="<<_S(_link_name)<<
    " WHERE l_id="<<_link_id;
    _p_dbase->qry(s);
    delete [] from;


    s.str("");
    s << "UPDATE wss SET" <<
    " nlks="<<_web_site->_total_links<<
    " WHERE s_id="<<_web_site->_site_id;
    _p_dbase->qry(s);

    // cout << "[[\n";
    // cout << _compressed <<"\n";
    // cout << "]]\n";
    _compressed.clear();

    uint32_t    kid = 0;
    char        tblname[16];
    const char* wordd;

    map<string,int32_t>::const_iterator b =  _scores.begin();
    for(; b!=_scores.end(); ++b)
    {
        wordd=b->first.c_str();
        // cout << b->first << "\t" << b->second << "\n";
        s.str("");
//        const char* bf = b->first.c_str();
        s << "SELECT kw_id FROM kws WHERE wrd='"<<b->first<<"';";
        SR rezs(_p_dbase->qry(s, true));
        size_t nrows = rezs->rows();
        if(nrows>0)
        {
            kid = _U(rezs->fetch(0));
        }
        else
        {
            s.str("");
            s << "INSERT INTO kws (wrd) values ('"<<b->first<<"');";
            //      // cout << s << "\n";
            _p_dbase->qry(s);
            s.str("");
            s << "SELECT kw_id FROM kws WHERE wrd='"<<b->first<<"';";
            SR rezs(_p_dbase->qry(s, true));
//            size_t nrows = rezs->rows();

            kid = _p_dbase->lastid();
        }
        // distrib 0-20
        int32_t tid = __calc_wkey(b->first);
        ::sprintf(tblname,"kwds%d",tid);
        s.str("");
        s << "INSERT INTO "<<tblname<<" (l_id, kw_id, wg, s_id) values ("<<
        _link_id <<
        "," << kid <<
        "," << b->second <<
        "," << _web_site->_site_id <<");";
        //    // cout << s << "\n";
        _p_dbase->qry(s);
    }
    _scores.clear();

    return _p_dbase->err();

}

static bool __good_host(const uripage *link_page)
{
    if((link_page->_uri.find("(")!=string::npos && link_page->_uri.find(")")!=string::npos) ||
       (link_page->_uri.find(";")!=string::npos))
       {
           GLOGT("BAD LINK" << link_page->_uri);
           return false;
       }
    return true;
}

bool _same_host(string h1, string h2)
{
    pcrecpp::RE("^http://").GlobalReplace("", &h1);
    pcrecpp::RE("^https://").GlobalReplace("", &h1);
    pcrecpp::RE("^www.").GlobalReplace("", &h1);

    pcrecpp::RE("^http://").GlobalReplace("", &h2);
    pcrecpp::RE("^https://").GlobalReplace("", &h2);
    pcrecpp::RE("^www.").GlobalReplace("", &h2);

    return h1==h2;
}


void uripage::_enqueue(string uri, const string& name, uint32_t sid, uint32_t dpth, int32_t parentid)
{
    if( dpth >_web_site->_depth_max || _web_site->_total_links > _web_site->_max_links || __bad_link(uri))
    {
        return;
    }

    pcrecpp::RE("(\\?|&)[0-9a-zA-Z]+(oscsid|osCsid|SID|sid|s|S)=[0-9a-zA-Z]+").GlobalReplace("", &uri);
    pcrecpp::RE("$\\?").GlobalReplace("", &uri);
    pcrecpp::RE("#+").GlobalReplace("", &uri);
    pcrecpp::RE("^/").GlobalReplace("", &uri);
    pcrecpp::RE("index\.(php|html|asp|htm)$").GlobalReplace("", &uri);

    if(uri.empty() || glb::inset(uri) )
    {
        GLOGT("Q[X]<--" << uri);
        return;
    }
     GLOGT("Q<---" << uri << "@" << _host);

//    assert(_link_id>0);
    uripage *link_page = new uripage(_web_site, uri, this);

    if(link_page->_host == _host && __good_host(link_page)) // no externals
    {

        link_page->_level_down  = dpth;
        link_page->_parent_id   = parentid;
        link_page->_link_name   = name.empty() ? "index" : name;
        if(glb::enqueue(link_page))
        {
            ++_links_count;
            ++_web_site->_total_links;
        }
        else
            delete link_page;
    }
    else
        delete link_page;
}

int32_t uripage::_score(const char* tag)const
{
    static struct TgScore
    {
        char tagL[16];
        int32_t score;
    } tgss[]=
    {
        {"h1", 5000},
        {"h2", 4000},
        {"h3", 3000},
        {"h4", 2000},
        {"h5", 1000},
        {"href", 200},
        {"a", 200},
        {"src", 2000},
        {"img", 2000},
        {"tit", 5000},
        {"meta", 5000},
        {"b", 400},
        {"strong", 400},
        {"",0}
    };

    float score = 10.0;

    for(int k=0; tgss[k].score; ++k)
    {
        if(!strcmp(tag, tgss[k].tagL))
        {
            score = tgss[k].score;
            break;
        }
    }
    score *= 1.0/(1+_level_down);
    return (int32_t)score;
    /*
       ($words_in_page + $word_in_title * $title_weight +
    			  $word_in_domain * $domain_weight +
    			  $word_in_path * $path_weight + $meta_keyword * $meta_weight) *10 / (0.8 +0.2*$path_depth);
    */
}

inline bool __word_ok(const string& word)
{
    static string reject="ok in at to eh duh ha yes this wmv tar"
                         "that where here not or and under var gz"
                         "then time other near by lol own org"
                         "contact mail website if out dif href html asp gif png img net"
                         "web w3c with such because neither either way "
                         "when well who what will do done which was there here because tell top"
                         "var int function copy http zip php pdf"
                         "www size height width while above below"
                         "under you me his our their him mine your upper left right from into";
    if(word.size()<3)
        return false;
    if(reject.find(word)!=string::npos)
        return false;
     return true;

}

inline bool __exp_ok(const string& word)
{
    if(word.size()<3)
        return false;

    if(word.find("<")!=string::npos && word.find(">")!=string::npos)
        return false;
    if(word.find("{")!=string::npos && word.find("}")!=string::npos)
        return false;
    return true;
}


void uripage::_add_word(string word, int32_t score, bool expression, bool incontent)
{
    if(!__exp_ok(word))
        return;


    pcrecpp::RE("/[^a-zA-Z0-9]+/").GlobalReplace(" ", &word);
    pcrecpp::RE("[|\-\/:\"\'\(\)<>{}\t;]").GlobalReplace(" ", &word);
    pcrecpp::RE("[/\\\\/]").GlobalReplace(" ", &word);
    pcrecpp::RE("&(quote|amp|nbsp)").GlobalReplace(" ", &word);
    pcrecpp::RE("[\r\n]").GlobalReplace("", &word);
    string local=word;
    pcrecpp::RE("\\'").GlobalReplace("", &local);
    pcrecpp::RE("[\ \]+").GlobalReplace(" ", &local);

    if(local.length()<3)
        return;

    if(local.find(' ')!=string::npos )
    {
        std::stringstream strstr(local);

        std::istream_iterator<std::string> it(strstr);
        std::istream_iterator<std::string> end;
        std::vector<std::string> result(it, end);

        std::vector<std::string>::iterator b = result.begin();
        for(; b!= result.end(); ++b)
        {
            const string& word = *b;
            if(__word_ok(word)  && _scores[word] < 10000)
            {
                int addon=0;
                map<string,int32_t>::iterator it = _web_site->_scores.find(word);
                if(it != _web_site->_scores.end())
                {
                    addon = it->second;
                }

                //mysql_real_escape_string(&_p_dbase->_mysql, loco, word.c_str(), word.length());
                _scores[word]=score + addon;
                if(score > _DOMAIN_SCORE)
                {
                    _web_site->_scores[word] = (score/100);
                }
                GLOGD( "<-:" << word << " / " << score);
            }
        }
    }
    else if(__word_ok(local))
    {
       // mysql_real_escape_string(&_p_dbase->_mysql, loco, word.c_str(), word.length());
        _scores[local] = score;
        if(score > _DOMAIN_SCORE)
        {
            _web_site->_scores[local]=score;
            GLOGD( "<-:" << word << " / " << score);

        }
    }
    if(_accum < GCFG->_mitza.pagelen)
    {
        _compressed+= local+" ";
        _accum += local.length() + 1;
    }
}

inline char _tolow(char in){
  if(in<='Z' && in>='A')
    return in-('Z'-'z');
  return in;
}

//std::transform(data.begin(), data.end(), data.begin(), _tolow);

void uripage::_parse_page()
{
    GLOGI( ">>>>[[" << _uri << "]]<<<<");

    if(_content.empty() || _db_update_link())
    {
        GLOGE("error updating link");
        return;
    }

    HTML::ParserDom parser;
    tree<HTML::Node> dom = parser.parseTree(_content);
    _content.clear();

    //Dump all nlks in the tree
    bool                        content=false;
    int32_t                     score = 0;
    string                      tag,link,val,attr,name,ilink;
    tree<HTML::Node>::iterator  it = dom.begin();
    tree<HTML::Node>::iterator  end = dom.end();
    for (; it != end; ++it)
    {
        if(it->isTag())
        {

            tag = it->tagName();
            std::transform(tag.begin(), tag.end(), tag.begin(), _tolow);
            score+=_score(tag.c_str());

            if((tag=="a") || (tag=="meta") || (tag=="img"))
            {
                it->parseAttributes();
                const std::map<std::string, std::string>& attrs = it->attributes();
                if(attrs.size())
                {
                    std::map<std::string, std::string>::const_iterator b = attrs.begin();
                    for(; b!=attrs.end(); b++)
                    {
                        attr=b->first;
                        std::transform(attr.begin(), attr.end(), attr.begin(), _tolow);
                        if(tag=="img" && b->first=="src")
                        {
                            ilink=b->second;
                            content=false;
                        }
                        else if(tag=="img" && b->first=="alt")
                        {
                            val=b->second;
                            content=true;
                        }
                        else if(tag=="a" &&   b->first=="href")
                        {
                            link=b->second;
                            content=true;
                        }
                        if(tag=="meta" && b->first=="description")
                        {
                            val=b->second;
                            content=false;
                        }
                        if(tag=="meta" && b->first=="keywords")
                        {
                            val=b->second;
                            content=false;
                        }
                        if(tag=="meta" && b->first=="http-equiv")
                        {
                            //not a freking shit library works properly in linuxoze messs
                            val = it->text();
                            size_t p =val.find("url=");
                            if(p!=string::npos)
                            {
                                val=val.substr(p+4);  // http://www.airsoftdepot.ca/catalog
                                p=val.find("\"");
                                if(p==string::npos)
                                    p=val.find("\'");
                                if(p!=string::npos)
                                {
                                    val=val.substr(0,p);
                                    if(val!=_uri)
                                        _enqueue(val, val, _web_site->_z_id, _level_down+1, _link_id);
                                }
                            }
                            content=false;
                        }
                    }
                }
            }

        }
        else
        {

            if(tag=="title")
            {
                _title=it->text(); //this page tit
                content=true;
            }
            if(tag=="meta" && attr=="name")
            {
                _desc=it->text(); //this page desc
                content=false;
            }
            if(tag=="a" && attr=="href")
            {
                name=it->text();  // next page link name
                content=false;
            }
            if(val.empty())
            {
                val = it->text();
                content=true;
            }

            if(!link.empty() && link[0]!='#' && link!="/")
            {
                _enqueue(link, name.empty() ? link : name, _web_site->_z_id, _level_down+1, _link_id);
                if(!name.empty())
                    _add_word(name, score, true/*is a name*/, content);
            }
            else if(!val.empty())
            {
                if(tag!="script")
                    _add_word(val, score, false, content);
            }

            val.clear();
            link.clear();
            tag.clear();
            score=0;
            content=false;
        }
    }
    _db_finalize_page();
}

void wsite::_db_clean_unrefs()
{
    AutoLock __l(_m);
    stringstream s;
    s << "DELETE FROM lnks WHERE s_id="<<_site_id << " AND z_id != "<<_z_id <<";" ;
    _db->qry(s);
}
